{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src='http://hilpisch.com/taim_logo.png' width=\"350px\" align=\"right\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Artificial Intelligence in Finance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Machine Learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dr Yves J Hilpisch | The AI Machine\n",
    "\n",
    "http://aimachine.io | http://twitter.com/dyjh"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<blockquote>\"A computer program is said to learn from experience 𝐸 with respect to some class of tasks 𝑇 and performance measure 𝑃, if its performance at tasks in 𝑇, as measured by 𝑃, improves with experience 𝐸.\" — Mitchell (1997)</blockquote>\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pylab import plt, mpl\n",
    "np.random.seed(100)\n",
    "plt.style.use('seaborn')\n",
    "mpl.rcParams['savefig.dpi'] = 300\n",
    "mpl.rcParams['font.family'] = 'serif'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'http://hilpisch.com/aiif_eikon_eod_data.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw = pd.read_csv(url, index_col=0, parse_dates=True)['EUR=']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "l = raw.resample('1M').last()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "l.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "l.plot(figsize=(10, 6), title='EUR/USD monthly');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "l = l.values\n",
    "l -= l.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = np.linspace(-2, 2, len(l))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10, 6))\n",
    "plt.plot(f, l, 'ro')\n",
    "plt.title('Sample Data Set')\n",
    "plt.xlabel('features')\n",
    "plt.ylabel('labels');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Success"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def MSE(l, p):\n",
    "    return np.mean((l - p) ** 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reg = np.polyfit(f, l, deg=5)\n",
    "reg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = np.polyval(reg, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MSE(l, p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10, 6))\n",
    "plt.plot(f, l, 'ro', label='sample data')\n",
    "plt.plot(f, p, '--', label='regression')\n",
    "plt.legend();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "for i in range(10, len(f) + 1, 20):\n",
    "    reg = np.polyfit(f[:i], l[:i], deg=3)\n",
    "    p = np.polyval(reg, f)\n",
    "    mse = MSE(l, p)\n",
    "    print(f'{i:3d} | MSE={mse}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "import tensorflow as tf\n",
    "tf.random.set_seed(100)\n",
    "tf.get_logger().setLevel(logging.ERROR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Dense\n",
    "from keras.models import Sequential"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()  \n",
    "model.add(Dense(256, activation='relu', input_dim=1))\n",
    "model.add(Dense(1, activation='linear'))\n",
    "model.compile(loss='mse', optimizer='rmsprop')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%time h = model.fit(f, l, epochs=1500, verbose=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = model.predict(f).flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MSE(l, p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10, 6))\n",
    "plt.plot(f, l, 'ro', label='sample data')\n",
    "plt.plot(f, p, '--', label='DNN approximation')\n",
    "plt.legend();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = pd.DataFrame(h.history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res.iloc[100:].plot(figsize=(10, 6))\n",
    "plt.ylabel('MSE')\n",
    "plt.xlabel('epochs');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Capacity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reg = {}\n",
    "for d in range(1, 12, 2):\n",
    "    reg[d] = np.polyfit(f, l, deg=d)\n",
    "    p = np.polyval(reg[d], f)\n",
    "    mse = MSE(l, p)\n",
    "    print(f'{d:2d} | MSE={mse}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10, 6))\n",
    "plt.plot(f, l, 'ro', label='sample data')\n",
    "for d in reg:\n",
    "    p = np.polyval(reg[d], f)\n",
    "    plt.plot(f, p, '--', label=f'deg={d}')\n",
    "plt.legend();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_dnn_model(hl=1, hu=256):\n",
    "    ''' Function to create Keras DNN model.\n",
    "    \n",
    "    Parameters\n",
    "    ==========\n",
    "    hl: int\n",
    "        number of hidden layers\n",
    "    hu: int\n",
    "        number of hidden units (per layer)\n",
    "    '''\n",
    "    model = Sequential()\n",
    "    for _ in range(hl):\n",
    "        model.add(Dense(hu, activation='relu', input_dim=1))\n",
    "    model.add(Dense(1, activation='linear'))\n",
    "    model.compile(loss='mse', optimizer='rmsprop')\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = create_dnn_model(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%time model.fit(f, l, epochs=2500, verbose=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = model.predict(f).flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MSE(l, p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10, 6))\n",
    "plt.plot(f, l, 'r', label='sample data')\n",
    "plt.plot(f, p, '--', label='DNN approximation')\n",
    "plt.legend();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "te = int(0.25 * len(f))\n",
    "va = int(0.25 * len(f))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(100)\n",
    "ind = np.arange(len(f))\n",
    "np.random.shuffle(ind)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ind_te = np.sort(ind[:te])\n",
    "ind_va = np.sort(ind[te:te + va])\n",
    "ind_tr = np.sort(ind[te + va:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f_te = f[ind_te]\n",
    "f_va = f[ind_va]\n",
    "f_tr = f[ind_tr]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "l_te = l[ind_te]\n",
    "l_va = l[ind_va]\n",
    "l_tr = l[ind_tr]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reg = {}\n",
    "mse = {}\n",
    "for d in range(1, 22, 4):\n",
    "    reg[d] = np.polyfit(f_tr, l_tr, deg=d)\n",
    "    p = np.polyval(reg[d], f_tr)\n",
    "    mse_tr = MSE(l_tr, p)\n",
    "    p = np.polyval(reg[d], f_va)\n",
    "    mse_va = MSE(l_va, p)\n",
    "    mse[d] = (mse_tr, mse_va)\n",
    "    print(f'{d:2d} | MSE_tr={mse_tr:7.5f} | MSE_va={mse_va:7.5f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(2, 1, figsize=(10, 8), sharex=True)\n",
    "ax[0].plot(f_tr, l_tr, 'ro', label='training data')\n",
    "ax[1].plot(f_va, l_va, 'go', label='validation data')\n",
    "for d in reg:\n",
    "    p = np.polyval(reg[d], f_tr)\n",
    "    ax[0].plot(f_tr, p, '--', label=f'deg={d} (tr)')\n",
    "    p = np.polyval(reg[d], f_va)\n",
    "    plt.plot(f_va, p, '--', label=f'deg={d} (va)')\n",
    "ax[0].legend()\n",
    "ax[1].legend();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.callbacks import EarlyStopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = create_dnn_model(2, 256)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "callbacks = [EarlyStopping(monitor='loss',\n",
    "                           patience=100,\n",
    "                          restore_best_weights=True)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "h = model.fit(f_tr, l_tr, epochs=3000, verbose=False,\n",
    "          validation_data=(f_va, l_va),\n",
    "          callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(2, 1, sharex=True, figsize=(10, 8))\n",
    "ax[0].plot(f_tr, l_tr, 'ro', label='training data')\n",
    "p = model.predict(f_tr)\n",
    "ax[0].plot(f_tr, p, '--', label=f'DNN (tr)')\n",
    "ax[0].legend()\n",
    "ax[1].plot(f_va, l_va, 'go', label='validation data')\n",
    "p = model.predict(f_va)\n",
    "ax[1].plot(f_va, p, '--', label=f'DNN (va)')\n",
    "ax[1].legend();"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "res = pd.DataFrame(h.history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res.iloc[35::25].plot(figsize=(10, 6))\n",
    "plt.ylabel('MSE')\n",
    "plt.xlabel('epochs');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "p_ols = np.polyval(reg[5], f_te)\n",
    "p_dnn = model.predict(f_te).flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MSE(l_te, p_ols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MSE(l_te, p_dnn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10, 6))\n",
    "plt.plot(f_te, l_te, 'ro', label='test data')\n",
    "plt.plot(f_te, p_ols, '--', label='OLS prediction')\n",
    "plt.plot(f_te, p_dnn, '-.', label='DNN prediction');\n",
    "plt.legend();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bias & Variance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f_tr = f[:20:2]\n",
    "l_tr = l[:20:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f_va = f[1:20:2]\n",
    "l_va = l[1:20:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reg_b = np.polyfit(f_tr, l_tr, deg=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reg_v = np.polyfit(f_tr, l_tr, deg=9, full=True)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f_ = np.linspace(f_tr.min(), f_va.max(), 75)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10, 6))\n",
    "plt.plot(f_tr, l_tr, 'ro', label='training data')\n",
    "plt.plot(f_va, l_va, 'go', label='validation data')\n",
    "plt.plot(f_, np.polyval(reg_b, f_), '--', label='high bias')\n",
    "plt.plot(f_, np.polyval(reg_v, f_), '--', label='high variance')\n",
    "plt.ylim(-0.2)\n",
    "plt.legend(loc=2);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import r2_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(reg, f, l):\n",
    "    p = np.polyval(reg, f)\n",
    "    bias = np.abs(l - p).mean()\n",
    "    var = p.var()\n",
    "    msg = f'MSE={MSE(l, p):.4f} | R2={r2_score(l, p):9.4f} | '\n",
    "    msg += f'bias={bias:.4f} | var={var:.4f}'\n",
    "    print(msg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluate(reg_b, f_tr, l_tr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluate(reg_b, f_va, l_va)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluate(reg_v, f_tr, l_tr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluate(reg_v, f_va, l_va)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cross-Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.pipeline import make_pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def PolynomialRegression(degree=None, **kwargs):\n",
    "    return make_pipeline(PolynomialFeatures(degree),\n",
    "                        LinearRegression(**kwargs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.set_printoptions(suppress=True,\n",
    "        formatter={'float': lambda x: f'{x:12.2f}'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('\\nCross-validation scores')\n",
    "print(74 * '=')\n",
    "for deg in range(0, 10, 1):\n",
    "    model = PolynomialRegression(deg)\n",
    "    cvs = cross_val_score(model, f.reshape(-1, 1), l, cv=5)\n",
    "    print(f'deg={deg} | ' + str(cvs.round(2)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(100)\n",
    "tf.random.set_seed(100)\n",
    "from keras.wrappers.scikit_learn import KerasRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = KerasRegressor(build_fn=create_dnn_model,\n",
    "                      verbose=False, epochs=1000,\n",
    "                      hl=1, hu=36)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%time cross_val_score(model, f, l, cv=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = KerasRegressor(build_fn=create_dnn_model,\n",
    "                      verbose=False, epochs=1000,\n",
    "                      hl=3, hu=256)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%time cross_val_score(model, f, l, cv=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src='http://hilpisch.com/taim_logo.png' width=\"350px\" align=\"right\">\n",
    "\n",
    "<br><br><br><a href=\"http://tpq.io\" target=\"_blank\">http://tpq.io</a> | <a href=\"http://twitter.com/dyjh\" target=\"_blank\">@dyjh</a> | <a href=\"mailto:ai@tpq.io\">ai@tpq.io</a>"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
